﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using OpenNLP.Tools.Chunker;
using OpenNLP.Tools.Parser;
using OpenNLP.Tools.PosTagger;
using OpenNLP.Tools.SentenceDetect;
using OpenNLP.Tools.Tokenize;
using OpenNLP.Tools.Trees;
using SharpEntropy;
using SharpEntropy.IO;

namespace Test
{
    internal class Program
    {
        private static readonly string currentDirectory = Environment.CurrentDirectory + "/../../";

        private static void Main(string[] args)
        {
            /*FileStream ostrm;
            StreamWriter writer;
            TextWriter oldOut = Console.Out;
            try
            {
                ostrm = new FileStream("C:\\Users\\Alexandre\\Desktop\\vs_output_2.txt", FileMode.OpenOrCreate, FileAccess.Write);
                writer = new StreamWriter(ostrm);
            }
            catch (Exception e)
            {
                Console.WriteLine("Cannot open Redirect.txt for writing");
                Console.WriteLine(e.Message);
                return;
            }
            Console.SetOut(writer);*/


            /*// read file
            var tokenizerTrainingFilePath = currentDirectory + "Input/tokenizer.train";
            var outputFilePath = currentDirectory + "Output/EnglishTok.nbin";
            MaximumEntropyTokenizer.Train(tokenizerTrainingFilePath, outputFilePath);*/

            // test detokenization
            /*var tokens = new List<string>() {"do", "n't", "commit"};
            var detokenizer = new DictionaryDetokenizer();
            var result = detokenizer.Detokenize(tokens.ToArray());
            Console.WriteLine(result);*/

            /*// train model file
            var inputFilePath = currentDirectory + "Input/sentences.train";
            var outputFilePath = currentDirectory + "Output/" + Path.GetFileNameWithoutExtension(inputFilePath) + ".nbin";
            var iterations = 100;
            var cut = 5;
            var endOfSentenceScanner = new CharactersSpecificEndOfSentenceScanner();
            Console.WriteLine("Training model...");
            var model = MaximumEntropySentenceDetector.TrainModel(inputFilePath, iterations, cut, endOfSentenceScanner);
            Console.WriteLine("Writing output file '{0}'...", outputFilePath);
            new BinaryGisModelWriter().Persist(model, outputFilePath);
            Console.WriteLine("Output file written.");*/

            // detect tokenization issues
            var pathToFile = currentDirectory + "Input/tokenizerIssues.txt";
            var modelPath = currentDirectory + "../Resources/Models/";
            var tokenizer = new EnglishRuleBasedTokenizer();
            //var tokenizer = new EnglishMaximumEntropyTokenizer(modelPath + "EnglishTok.nbin");
            var allLines = File.ReadAllLines(pathToFile);

            var stopwatch = Stopwatch.StartNew();
            for (int i = 0; i < 1000; i++)
            {
                foreach (var line in allLines)
                {
                    var tokens = tokenizer.Tokenize(line);
                    //Console.WriteLine(string.Join(" | ", tokens));
                }
            }
            stopwatch.Stop();
            Console.WriteLine(stopwatch.Elapsed.ToString("g"));

            // parsing
            //var sentence = "If she wakes up, and she's a little more hurt, or she's a little more concerned that she might not make it out of there in one piece, and I had something to pick a lock with...";
            //var sentence = "that you can do what you set out to do";
            //var sentence = "This is a test.";
            /*var sentence = "The kids actually get involved in mapping out where the community center should be";
            var tokenizer = new EnglishMaximumEntropyTokenizer(currentDirectory + "../Resources/Models/EnglishTok.nbin");
            var tokens = tokenizer.Tokenize(sentence);
            var modelPath = currentDirectory + "../Resources/Models/";
            var parser = new EnglishTreebankParser(modelPath, true, false);
            var parse = parser.DoParse(tokens);
            // Extract dependencies from lexical tree
            var tlp = new PennTreebankLanguagePack();
            var gsf = tlp.GrammaticalStructureFactory();
            var tree = new ParseTree(parse);
            Console.WriteLine(tree);
            var gs = gsf.NewGrammaticalStructure(tree);
            var dependencies = gs.TypedDependencies();

            foreach (var dep in dependencies)
            {
                Console.WriteLine(dep);
            }

            
            Console.WriteLine("===========");
            Console.WriteLine("OK");
            Console.ReadKey();*/

            /*var lemmas = new List<string>()
            {
                "hang up",
                "barge into",
                "pack up",
                "lie down",
                "slip by",
                "water down",
                "beef up",
                "rack up",
                "blow up",
                "cheat on",
                "screw up",
                "angle for",
                "hang in",
                "tick off",
                "wipe away",
                "account for",
                "get rid of",
                "turn up",
                "bow down",
                "come between",
                "tie up",
                "press on",
                "tear down",
                "load up",
                "lie with",
                "wash down",
                "piss off",
                "hack off",
                "tear off",
                "see to",
                "cave in",
                "coop up",
                "sneak off",
                "stick with",
                "melt down",
                "go forth",
                "pile up",
                "hunt down",
                "sober up",
                "slip out",
                "shut in",
                "come down",
                "lord it over",
                "prey on",
                "shape up",
                "chip away",
                "carry out",
                "tense up",
                "live up to",
                "settle for",
                "cough up",
                "spread out",
                "hand over",
                "frown upon",
                "sew up",
                "fawn over",
                "toy with",
                "snap out of",
                "light up",
                "wade through",
                "verge on",
                "settle down",
                "ferret out",
                "drum up",
                "wind up",
                "stand for",
                "cast aside",
                "batten down",
                "go up against",
                "flag down",
                "come up with",
                "turn in",
                "sweep up",
                "whack off",
                "sum up",
                "back down",
                "bargain for",
                "flip out",
                "dig",
                "grind",
                "scratch",
                "paw",
                "edge",
                "stumble",
                "spill",
                "whole",
                "cradle",
                "bail",
                "onesy",
                "fill",
                "pull",
                "loop",
                "uptight",
                "towel",
                "bust",
                "condom",
                "honestly",
                "wear",
                "bathroom",
                "sit",
                "glad",
                "stork",
                "lease",
                "tear",
                "widely",
                "acknowledge",
                "fact",
                "quite",
                "enough",
                "drown",
                "whistle",
                "groundskeeper",
                "perhaps",
                "purpose",
                "bunch",
                "snot",
                "maggot",
                "pick",
                "wish",
                "ahead",
                "tale",
                "once",
                "upon",
                "leave",
                "belief",
                "fool",
                "amaze",
                "pretty",
                "cattle",
                "above",
                "either",
                "besides",
                "keel",
                "over",
                "barely",
                "witness",
                "warm",
                "avoid",
                "whose",
                "robbery",
                "neighborhood",
                "several",
                "sell",
                "head",
                "seclude",
                "slum",
                "necklace",
                "least",
                "string",
                "occurrence",
                "dump",
                "starve",
                "though",
                "screw",
                "poor",
                "raise",
                "sleeve",
                "any",
                "smuggle",
                "still",
                "dirt",
                "shame",
                "target",
                "trade",
                "guilt",
                "pity",
                "way",
                "shortly",
                "hold",
                "steel",
                "briefcase",
                "reach",
                "sinner",
                "injury",
                "include",
                "old",
                "wisdom",
                "jam",
                "relieve",
                "adjourn",
                "gruesome",
                "park",
                "taillight",
                "dripping",
                "rear",
                "parolee",
                "commit",
                "relative",
                "hose",
                "weird",
                "beard",
                "should",
                "shave",
                "cocky",
                "vacutainer",
                "roger",
                "shift",
                "clue",
                "streak",
                "stroll",
                "fancy",
                "wreak",
                "utter",
                "folder",
                "lousy",
                "pride",
                "suit",
                "might",
                "heel",
                "earn",
                "meantime",
                "bite",
                "tooth",
                "folklore",
                "convenience",
                "horse",
                "dairy",
                "shamble",
                "stick",
                "behalf",
                "shell",
                "forth",
                "downtown",
                "craving",
                "hence",
                "subsidiary",
                "hire",
                "snort",
                "janitor",
                "run",
                "reliable",
                "vigorously",
                "pursue",
                "primarily",
                "weak",
                "hit",
                "steal",
                "haul",
                "hobo",
                "rest",
                "elusive",
                "dubbed",
                "regrow",
                "close",
                "uphold",
                "gifted",
                "spew",
                "foundle",
                "healthy",
                "perform",
                "decorum",
                "quote",
                "handsome",
                "kindness",
                "wad",
                "somehow",
                "fling",
                "eager",
                "diversion",
                "result",
                "freak",
                "faggot",
                "hide",
                "led",
                "rather",
                "rat",
                "taste",
                "shipment",
                "scarf",
                "step",
                "councilman",
                "worse",
                "surrender",
                "struggle",
                "try",
                "blanket",
                "seek",
                "holy",
                "magnificent",
                "benevolent",
                "congregation",
                "renowned",
                "renege",
                "petty",
                "grow",
                "hush",
                "sink",
                "sharp",
                "swap",
                "awkward",
                "meal",
                "vegetable",
                "utmost",
                "furthermore",
                "truce",
                "upstairs",
                "trash",
                "mercy",
                "early",
                "overwhelm",
                "impotent",
                "slip",
                "whatever",
                "stoner",
                "rethink",
                "spoil",
                "worthy",
                "row",
                "shear",
                "unsully",
                "nail",
                "knock",
                "cripple",
                "thrust",
                "twat",
                "stand",
                "trial",
                "demand",
                "wooden",
                "sooner",
                "guilty",
                "altogether",
                "wonder",
                "roommate",
                "annoy",
                "fulfill",
                "anchor",
                "pace",
                "opponent",
                "lately",
                "thick",
                "beg",
                "betray",
                "scent",
                "safeguard",
                "horn",
                "truck",
                "yell",
                "seatbelt",
                "custom",
                "libation",
                "runaway",
                "debt",
                "redeem",
                "frighten",
                "cast",
                "unless",
                "carryall",
                "sturdy",
                "fit",
                "overseer",
                "limb",
                "sound",
                "maid",
                "wide",
                "spark",
                "unrest",
                "illness",
                "flu",
                "truthful",
                "plot",
                "indeed",
                "summon",
                "spend",
                "pleasure",
                "seal",
                "heaven",
                "hunt",
                "coin",
                "skulk",
                "belong",
                "throat",
                "kind",
                "reward",
                "below",
                "land",
                "map",
                "hopefully",
                "fellow",
                "support",
                "accordingly",
                "overemotional",
                "rusty",
                "plough",
                "poppy",
                "dull",
                "wit",
                "realm",
                "assign",
                "punishment",
                "weight",
                "bear",
                "burden",
                "greed",
                "march",
                "fix",
                "bathe",
                "craft",
                "teach",
                "despise",
                "mount",
                "hospitable",
                "shy",
                "privacy",
                "properly",
                "exhaust",
                "attend",
                "worth",
                "gates",
                "beyond",
                "untie",
                "toward",
                "rot",
                "fly",
                "fart",
                "outlaw",
                "hardship",
                "climb",
                "fight",
                "alongside",
                "saddle",
                "repaid",
                "away",
                "forgiveness",
                "riddle",
                "fear",
                "merchant",
                "tongue",
                "blind",
                "piss",
                "ought",
                "damn",
                "hand",
                "bedazzle",
                "beat",
                "fall",
                "shield",
                "vain",
                "bow",
                "enrol",
                "loan",
                "deck",
                "shall",
                "nosedive",
                "knowingly",
                "waffle",
                "contempt",
                "aware",
                "may",
                "vault",
                "overseas",
                "thief",
                "shatter",
                "skip",
                "casualty",
                "dummy",
                "scuba",
                "dive",
                "unaware",
                "unzip",
                "snap",
                "heat",
                "accurate",
                "arsonist",
                "whisper",
                "beam",
                "unpack",
                "dare",
                "bout",
                "obvious",
                "messy",
                "tail",
                "smush",
                "pine",
                "peek",
                "harness",
                "whereby",
                "sight",
                "wind",
                "pee",
                "wheel",
                "needle",
                "bind",
                "devastate",
                "fruition",
                "merge",
                "game",
                "plummet",
                "mistrust",
                "thumb",
                "assume",
                "tilt",
                "nurture",
                "decipher",
                "stake",
                "trigger",
                "earthquake",
                "loss",
                "assess",
                "intend",
                "tinker",
                "collapse",
                "hazmat",
                "bunk",
                "slob",
                "toss",
                "pin",
                "sustain",
                "shorthanded",
                "subdue",
                "supply",
                "lack of",
                "breach",
                "sprinkler",
                "feel",
                "awful",
                "frank",
                "grab",
                "boastful",
                "check",
                "on account of",
                "fate",
                "twice",
                "brew",
                "overcome",
                "poverty",
                "behavior",
                "sweep",
                "match",
                "nickel",
                "draw",
                "sellout",
                "legit",
                "bullet",
                "weave",
                "relate",
                "undercut",
                "hook",
                "bury",
                "whip",
                "shape",
                "misbehave",
                "get worse",
                "upset",
                "entice",
                "silly",
                "med",
                "verse",
                "coverage",
                "worldwide",
                "brave",
                "bill",
                "all of a sudden",
                "leverage",
                "self-serving",
                "tribute",
                "hold that thought",
                "sake",
                "wheelchair",
                "colostomy",
                "trail",
                "concern",
                "riff",
                "harsh",
                "void",
                "lawyer",
                "mansion",
                "wishy-washy",
                "phony",
                "raider",
                "orphan",
                "disease",
                "stink",
                "tidal wave",
                "avert",
                "wormhole",
                "circumstance",
                "allow",
                "consider",
                "dozen",
                "fair",
                "spring",
                "spot",
                "odd",
                "disable",
                "deplete",
                "mere",
                "pep talk",
                "retask",
                "warden",
                "trick",
                "debatable",
                "strike",
                "ash",
                "cranky",
                "cell",
                "shrink",
                "effective",
                "lair",
                "lie",
                "bender",
                "besmirch",
                "stash",
                "booby-trap",
                "denizen",
                "tough",
                "broadcast",
                "borrow",
                "laptop",
                "pecan",
                "syrup",
                "rubbish",
                "blink",
                "lower",
                "dampen",
                "accountable",
                "otherwise",
                "imply",
                "leave of absence",
                "repress",
                "chill",
                "threat",
                "stare",
                "profound",
                "mist",
                "pissed",
                "path",
                "chopper",
                "withdraw",
                "injure",
                "delve",
                "core",
                "cruise ship",
                "rut",
                "junk",
                "cahoot",
                "commendable",
                "recipe",
                "thieve",
                "stab",
                "reinforcement",
                "unfair",
                "truthfully",
                "priceless",
                "dread",
                "nightmare",
                "outvote",
                "afford",
                "hear",
                "contempt of Congress",
                "sewer",
                "endeavor",
                "amongst",
                "headache",
                "owner",
                "buyer",
                "as a matter of fact",
                "oven",
                "glow",
                "peak",
                "volatility",
                "delightful",
                "render",
                "warrant",
                "callous",
                "wallow",
                "believable",
                "emit",
                "field hospital",
                "bouncer",
                "broom closet",
                "feature",
                "occur",
                "tip",
                "thrill",
                "ambush",
                "kraut",
                "stripe",
                "lewd",
                "noon",
                "pop",
                "tad",
                "conspicuous",
                "tire",
                "commute",
                "errand",
                "volunteer",
                "straightaway",
                "taint",
                "remain",
                "wristwatch",
                "trousers",
                "clothe",
                "lobby",
                "stairwell",
                "further",
                "merely",
                "although",
                "aptly",
                "record",
                "enter",
                "payroll",
                "call the shots",
                "few",
                "defenseless",
                "butler",
                "strain",
                "skinny-dip",
                "ace",
                "pussycat",
                "rent",
                "flat",
                "curfew",
                "haven",
                "demure",
                "scar",
                "corpse",
                "wreckage",
                "lift",
                "whether",
                "novel",
                "rug",
                "attorney",
                "sprout",
                "brush-off",
                "rhubarb pie",
                "harm",
                "manhole",
                "involvement",
                "strap",
                "spaniard",
                "within",
                "ensure",
                "jewish",
                "tie",
                "tailor",
                "yank",
                "egghead",
                "crate",
                "pull the trigger",
                "refill",
                "lad",
                "sigil",
                "skewer",
                "last",
                "blade",
                "dead set",
                "prone",
                "lecher",
                "feast",
                "dwarf",
                "grandchild",
                "oath",
                "omen",
                "crown",
                "spear",
                "hunter",
                "grim",
                "grass",
                "stalk",
                "poacher",
                "mug",
                "scream",
                "vow",
                "sewing needle",
                "wet nurse",
                "wander",
                "mighty",
                "hound",
                "butcher",
                "pet",
                "wisely",
                "hill",
                "invade",
                "coward",
                "sane",
                "raven",
                "breastplate",
                "hammer",
                "halfwit",
                "stutter",
                "linger",
                "shout",
                "bend",
                "knee",
                "plead",
                "reprieve",
                "chewy",
                "wildle",
                "goat",
                "supper",
                "flee",
                "feather bed",
                "despair",
                "stroke",
                "yield",
                "vanquish",
                "sail",
                "silk",
                "hatch",
                "stunt",
                "close to",
                "pack",
                "flood in",
                "dreadful",
                "riot",
                "vie",
                "sting",
                "glove",
                "grief",
                "selfless",
                "forsake",
                "inheritance",
                "untimely",
                "wise",
                "sniveling",
                "thirsty",
                "motto",
                "sow",
                "famed",
                "stag",
                "delay",
                "shudder",
                "fondness",
                "misgiving",
                "vile",
                "dangle",
                "muster",
                "inch",
                "pompous",
                "cunt",
                "scheme",
                "spite",
                "thunder",
                "hoof",
                "herd",
                "eel",
                "stew",
                "bald",
                "field",
                "granary",
                "fortnight",
                "brand",
                "recklessness",
                "worship",
                "ease",
                "in spite of",
                "steward",
                "fetch",
                "sheet",
                "drag",
                "deny",
                "heir",
                "cloak",
                "leash",
                "oaf",
                "spit",
                "sew",
                "melt",
                "bare",
                "deaf",
                "shove",
                "carve",
                "slit",
                "wield",
                "toll",
                "curse",
                "raper",
                "inn",
                "unaccustomed",
                "onward",
                "stale",
                "faith",
                "bid",
                "pledge",
                "fealty",
                "gown",
                "scatter",
                "warmth",
                "stallion",
                "stern",
                "gash",
                "ham",
                "pyre",
                "wheat",
                "beneath",
                "holdfast",
                "howl",
                "gap",
                "harmless",
                "crush",
                "eldest",
                "behead",
                "litter",
                "jaw",
                "cater",
                "copper",
                "ploughman",
                "paddle",
                "turnip",
                "poorly",
                "acquire",
                "mitigate",
                "squire",
                "pour",
                "bauble",
                "chop off",
                "repeatedly",
                "according to",
                "at once",
                "stillborn",
                "scullion",
                "bride",
                "snatch",
                "fang",
                "claw",
                "cub",
                "get one's money's worth",
                "vouch",
                "dusk",
                "dawn",
                "smuggler",
                "flock",
                "grieve",
                "swell",
                "bloat",
                "flood",
                "locksmith",
                "dumb",
                "unlike",
                "exalt",
                "shelter",
                "withstand",
                "fever",
                "tend",
                "upstart",
                "overreach",
                "sin",
                "crib",
                "plow",
                "sneer",
                "sour cherry",
                "piecrust",
                "teat",
                "crop",
                "stream",
                "hen",
                "wretched",
                "prevail",
                "ladder",
                "childhood",
                "overpower",
                "in store",
                "recklessly",
                "strife",
                "hereby",
                "slander",
                "split",
                "ashore",
                "above all",
                "ablaze",
                "tangle with",
                "at any rate",
                "cow",
                "even so",
                "breed",
                "pox",
                "cough",
                "whimper",
                "forbid",
                "bedside",
                "ill",
                "degenerate",
                "relief",
                "ashamed",
                "puddle",
                "ale",
                "averse",
                "tedious",
                "graceless",
                "quandary",
                "lunge",
                "wrestle",
                "get one's juices flowing",
                "blunder",
                "rubble",
                "hinder",
                "unattended",
                "outrageous",
                "wealth",
                "muddy",
                "pen",
                "overpraise",
                "bake",
                "pie",
                "tether",
                "godly",
                "grunt",
                "sack",
                "stir",
                "knot",
                "thoroughly",
                "rub",
                "lend",
                "defile",
                "resilient",
                "carving knife",
                "tempt",
                "stem",
                "resolve",
                "shoveling",
                "shipboard",
                "yearn",
                "grave",
                "bump",
                "mince",
                "suitor",
                "oink",
                "elbow",
                "molasses",
                "prowl",
                "swineherd",
                "crook",
                "rope",
                "breast",
                "thus",
                "storm",
                "suffice",
                "bowel",
                "ransom",
                "weep",
                "scrub",
                "worm",
                "flea",
                "crow",
                "leaf",
                "straw man",
                "chin",
                "umber",
                "wary",
                "make amends",
                "haggle",
                "overplay",
                "bloodshed",
                "doe",
                "thwart",
                "gape",
                "pit",
                "swallow",
                "cling",
                "sulk",
                "prickly",
                "scourge",
                "windmill",
                "blacken",
                "entertainment",
                "cheat",
                "wage",
                "lamb",
                "seep",
                "foul",
                "entertain",
                "sheath",
                "deed",
                "second thought",
                "sustenance",
                "regard",
                "selfish",
                "witty",
                "pomposity",
                "truss",
                "hooded",
                "phantom limb",
                "itch",
                "pipe",
                "wisp",
                "jolly",
                "weary",
                "tame",
                "abate",
                "disown",
                "scrawny",
                "collar",
                "drain",
                "talker",
                "boar",
                "slay",
                "bold",
                "peel",
                "flay",
                "crisp",
                "acre",
                "idle",
                "larder",
                "boil",
                "seagull",
                "stubborn",
                "afterward",
                "leap",
                "asunder",
                "heartfelt",
                "decree",
                "distasteful",
                "cowardly",
                "cupbearer",
                "holiness",
                "sinful",
                "gullible",
                "straightforward",
                "squirm",
                "sour",
                "easy pickings",
                "leisure time",
                "fold",
                "mutineer",
                "whomever",
                "leisurely",
                "innard",
                "chivalry",
                "gutter",
                "soak",
                "mourn",
                "shade",
                "abreast",
                "fade",
                "tremendous",
                "amount",
                "clutch",
                "shipload",
                "crossbow",
                "brothel",
                "confine",
                "lonesome",
                "doeskin",
                "emerge",
                "powder",
                "pinch",
                "mood",
                "unadulterated",
                "lurk",
                "nest",
                "fleabite",
                "mildly",
                "stretch",
                "bargain",
                "deem",
                "infant",
                "squalor",
                "foe",
                "hobble",
                "bounty",
                "enlist",
                "foreigner",
                "ruler",
                "tenure",
                "greyscale",
                "misery",
                "dung",
                "bystander",
                "noose",
                "mulish",
                "snigger",
                "maiden",
                "sparrow",
                "fond",
                "wheelhouse",
                "likelihood",
                "grayscale",
                "dime",
                "spook",
                "impale",
                "contraband",
                "assault",
                "tangle",
                "thorn",
                "orphanage",
                "foster family",
                "perceive",
                "favoritism",
                "undermine",
                "enforce",
                "batch",
                "awe",
                "empower",
                "apprentice",
                "dally",
                "apprenticeship",
                "tincture",
                "repel",
                "fairly",
                "spell",
                "warlock",
                "unreliable",
                "sip",
                "hatred",
                "dust",
                "wee",
                "wicked",
                "wealthy",
                "bugger",
                "filth",
                "rabble",
                "deformity",
                "boredom",
                "doll",
                "cheek",
                "outmaneuver",
                "grovel",
                "whelp",
                "risky",
                "resentment",
                "trippy",
                "baffle",
                "tool",
                "cease",
                "hyphenate",
                "tamper",
                "gut",
                "proposal",
                "waver",
                "hint",
                "reckon",
                "creepy",
                "cold shoulder",
                "roach",
                "flare",
                "uneventful",
                "falter",
                "inconvenience",
                "ecstatic",
                "manic",
                "demise",
                "dingy",
                "stray",
                "romp",
                "snag",
                "conceal",
                "rafter",
                "grant",
                "compel",
                "upward",
                "collide",
                "twofold",
                "luggage",
                "musty",
                "wind gauge",
                "retaliation",
                "crafty",
                "charter",
                "trout",
                "foremost",
                "summit",
                "tantamount",
                "invigorating",
                "bishop",
                "benefit",
                "neglect",
                "haymaker",
                "untenable",
                "harvest",
                "sheaf",
                "puke",
                "trip",
                "mettle",
                "exceed",
                "seed",
                "estate",
                "cesspool",
                "ooze",
                "graft",
                "wrist",
                "somewhat",
                "expectation",
                "inherit",
                "reinstatement",
                "volition",
                "prep",
                "safely",
                "reckless",
                "aim",
                "reconcile",
                "horse sense",
                "property",
                "vessel",
                "rave",
                "stage",
                "scale",
                "unworthiness",
                "foresee",
                "thoughtless",
                "cull",
                "yoke",
                "setback",
                "misguided",
                "wayward",
                "fair enough",
                "bitter",
                "bait",
                "divide",
                "sway",
                "hang",
                "stitch",
                "obsidian",
                "bond",
                "sullen",
                "churn",
                "closure",
                "outweigh",
                "usefulness",
                "liver",
                "enhancer",
                "herald",
                "moody",
                "elder",
                "spine",
                "hypothesis",
                "beacon",
                "bribe",
                "mislead",
                "lung",
                "ripple",
                "length",
                "cufflink",
                "leather",
                "tomboy",
                "keg",
                "fuzz",
                "caterer",
                "iron",
                "drawer",
                "painkiller",
                "prissy",
                "disgrace",
                "gig",
                "sorta",
                "toot",
                "lib",
                "circle",
                "halfway",
                "flaw",
                "preemptive",
                "trough",
                "narrow",
                "settlement",
                "gnat",
                "stride",
                "plague",
                "high tide",
                "strait",
                "nor",
                "enlighten",
                "nondisclosure",
                "willingly",
                "pill",
                "abstain",
                "stifle",
                "tow",
                "preoccupy",
                "safeword",
                "wallet",
                "income",
                "waitress",
                "faint",
                "abundantly",
                "page",
                "lung cancer",
                "cue",
                "doomsayer",
                "expound",
                "wanker",
                "coat",
                "ointment",
                "decency",
                "metaphor",
                "leftover",
                "dubious",
                "on second thought",
                "fence",
                "cheap",
                "forfeit",
                "deception",
                "feather",
                "ink",
                "blur",
                "mirror image",
                "bide one's time",
                "nerdy",
                "diaper",
                "rv",
                "crust",
                "infer",
                "pot",
                "earful",
                "moronic",
                "cluster",
                "pickup",
                "sham",
                "remotely",
                "conquistador",
                "upper",
                "downer",
                "wire",
                "in earnest",
                "attic",
                "pumpkin",
                "fairy godmother",
                "lure",
                "doom",
                "empress",
                "deity",
                "wretch",
                "dowry",
                "quotable",
                "pal",
                "circumspect",
                "leak",
                "hardware",
                "gaze",
                "randomly",
                "misdirection",
                "moral compass",
                "carbide",
                "tungsten",
                "burn the midnight oil",
                "stub",
                "toe",
                "bodily",
                "buckle",
                "apply",
                "gear",
                "oak",
                "penance",
                "pension fund",
                "blurry",
                "comb",
                "nut",
                "swab",
                "warehouse",
                "careless",
                "glad tidings",
                "extinguisher",
                "prohibit",
                "catchy",
                "genuine",
                "mercurial",
                "dupe",
                "shelf life",
                "assumption",
                "purport",
                "nitrogen",
                "chart",
                "graze",
                "asset",
                "shrapnel",
                "garner",
                "waterfront",
                "laundry",
                "befriend",
                "testify",
                "pet peeve",
                "bug",
                "spleen",
                "bruise",
                "puppy",
                "boost",
                "wreck",
                "railing",
                "vat",
                "scrap",
                "brag",
                "hideout",
                "cloud",
                "tollbooth",
                "bully",
                "seam",
                "squash",
                "roadblock",
                "dodge",
                "impede",
                "scope",
                "grid",
                "jack",
                "district attorney",
                "furlough",
                "pawn",
                "breakthrough",
                "facility",
                "yip",
                "notch",
                "nap",
                "mattress",
                "whammy",
                "CAT scan",
                "ping",
                "adamant",
                "hock",
                "commuter train",
                "conundrum",
                "fathom",
                "berate",
                "handy",
                "alike",
                "hangover",
                "embolden",
                "retaliate",
                "lame",
                "trim",
                "guzzle",
                "mink coat",
                "stingy",
                "outrage",
                "celibacy",
                "grade",
                "prenup",
                "convertible",
                "bee",
                "prior",
                "scramble",
                "fingertip",
                "poll",
                "grease",
                "ledge",
                "shed",
                "waive",
                "indict",
                "clean slate",
                "excuse",
                "delineation",
                "solely",
                "shuffle",
                "snowstorm",
                "startle",
                "wrath",
                "sorely",
                "dissent",
                "prune",
                "ascend",
                "apex",
                "unbearable",
                "amend",
                "custody",
                "hail",
                "disdain",
                "make no bones about",
                "brawl",
                "dome",
                "tin can",
                "relentless"
            };
            var stopwatch = new Stopwatch();

            // regex perfs
            Regex lettersOnlyRegex = new Regex("^[a-zA-Z]+$", RegexOptions.Compiled);
            stopwatch.Start();
            //var tokenizer = new EnglishRuleBasedTokenizer();
            for (int i = 0; i < 100; i++)
            {
                foreach (var lemma in lemmas)
                {
                    //var tokens = tokenizer.Tokenize(lemma);
                    var res = lettersOnlyRegex.IsMatch(lemma);
                    //Console.WriteLine(tokens);
                }
            }
            stopwatch.Stop();
            Console.WriteLine("Regex.IsMatch on {0} lemmas: {1}ms", lemmas.Count, stopwatch.ElapsedMilliseconds);

            // function perfs
            stopwatch.Start();
            //var tokenizer = new EnglishRuleBasedTokenizer();
            for (int i = 0; i < 100; i++)
            {
                foreach (var lemma in lemmas)
                {
                    //var tokens = tokenizer.Tokenize(lemma);
                    var res = IsLetterOnly(lemma);
                    //Console.WriteLine(tokens);
                } 
            }
            stopwatch.Stop();
            Console.WriteLine("IsLetterOnly function on {0} lemmas: {1}ms", lemmas.Count, stopwatch.ElapsedMilliseconds);*/

            Console.ReadKey();
        }

        private static bool IsLetterOnly(string token)
        {
            foreach (var c in token)
            {
                if (!char.IsLetter(c))
                {
                    return false;
                }
            }
            return true;
        }

        /*private void TestDechunk()
        {
            // detokenize
            var inputs = new string[]
            {
                "- Harry's your sister. - Look, what exactly am I supposed to be doing here?",
                "\"Piss off!\"",
                "- Sorry Mrs. Hudson, I'll skip the tea. Off out. - Both of you?",
                "I love playing half-life; that's just who I am!",
                "That's why I... have just begun to write a book.",
                "And they lived happily ever after...",
                "It's gonna be $1.5 sir."
            };

            // 
            var tokenizer = new EnglishMaximumEntropyTokenizer(currentDirectory + "../Resources/Models/EnglishTok.nbin");
            var chunker = new EnglishTreebankChunker(currentDirectory + "../Resources/Models/EnglishChunk.nbin");
            var dechunker = new RegexDictionaryDechunker();
            var detokienizer = new DictionaryDetokenizer();
            var englishPosPath = currentDirectory + "../Resources/Models/EnglishPOS.nbin";
            var tagDictPath = currentDirectory + "../Resources/Models/Parser/tagdict";
            var posTagger = new EnglishMaximumEntropyPosTagger(englishPosPath, tagDictPath);

            foreach (var input in inputs)
            {
                string[] tokens = tokenizer.Tokenize(input);
                string[] tags = posTagger.Tag(tokens);

                var chunks = chunker.GetChunks(tokens, tags);
                var chunksStrings = chunks
                    .Select(ch => detokienizer.Detokenize(ch.TaggedWords.Select(tw => tw.Word).ToArray()))
                    .ToArray();
                var output = dechunker.Dechunk(chunksStrings);
                Console.WriteLine("input: " + input);
                Console.WriteLine("chunks: " + string.Join(" | ", chunks));
                Console.WriteLine("ouput: " + output);
                Console.WriteLine("--");
            }
        }*/
    }
}